{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Task \n",
    "\n",
    "This notebook produces the income shocks parameters. For the U.S., we use data from the PSID and for Europe from the ECHP. The notebooks, Preparation PSID and Preparation ECHP prepare the datasets used for this task. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "from scipy.optimize import minimize\n",
    "from functools import partial\n",
    "from statsmodels.formula.api import ols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We load the ECHP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>wave</th>\n",
       "      <th>year</th>\n",
       "      <th>age</th>\n",
       "      <th>co</th>\n",
       "      <th>Y</th>\n",
       "      <th>logY</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>country</th>\n",
       "      <th>hid</th>\n",
       "      <th>pid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2</th>\n",
       "      <th>7601</th>\n",
       "      <th>76101</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>DK</td>\n",
       "      <td>0.827465</td>\n",
       "      <td>-0.189389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8401</th>\n",
       "      <th>84101</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>DK</td>\n",
       "      <td>1.079543</td>\n",
       "      <td>0.076538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10701</th>\n",
       "      <th>107101</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>DK</td>\n",
       "      <td>0.767751</td>\n",
       "      <td>-0.264290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16501</th>\n",
       "      <th>165101</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>DK</td>\n",
       "      <td>1.332184</td>\n",
       "      <td>0.286820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19901</th>\n",
       "      <th>199101</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>DK</td>\n",
       "      <td>0.818934</td>\n",
       "      <td>-0.199751</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      wave    year   age  co         Y      logY\n",
       "country hid   pid                                               \n",
       "2       7601  76101    1.0  1994.0  42.0  DK  0.827465 -0.189389\n",
       "        8401  84101    1.0  1994.0  70.0  DK  1.079543  0.076538\n",
       "        10701 107101   1.0  1994.0  22.0  DK  0.767751 -0.264290\n",
       "        16501 165101   1.0  1994.0  46.0  DK  1.332184  0.286820\n",
       "        19901 199101   1.0  1994.0  23.0  DK  0.818934 -0.199751"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_pickle('../data_sources/echp/echp_incomes.pkl')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We subselect the age range for this task. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wave</th>\n",
       "      <th>year</th>\n",
       "      <th>age</th>\n",
       "      <th>Y</th>\n",
       "      <th>logY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>49335.000000</td>\n",
       "      <td>49335.000000</td>\n",
       "      <td>49335.000000</td>\n",
       "      <td>49335.000000</td>\n",
       "      <td>49335.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>4.338218</td>\n",
       "      <td>1999.642059</td>\n",
       "      <td>57.551191</td>\n",
       "      <td>1.022995</td>\n",
       "      <td>-0.139069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.307135</td>\n",
       "      <td>7.935081</td>\n",
       "      <td>18.349595</td>\n",
       "      <td>0.587244</td>\n",
       "      <td>0.619135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1994.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>0.012390</td>\n",
       "      <td>-4.390870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>1995.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>0.632637</td>\n",
       "      <td>-0.457858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>1997.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>0.896917</td>\n",
       "      <td>-0.108792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>1999.000000</td>\n",
       "      <td>73.000000</td>\n",
       "      <td>1.256659</td>\n",
       "      <td>0.228456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8.000000</td>\n",
       "      <td>2021.000000</td>\n",
       "      <td>85.000000</td>\n",
       "      <td>4.053713</td>\n",
       "      <td>1.399633</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               wave          year           age             Y          logY\n",
       "count  49335.000000  49335.000000  49335.000000  49335.000000  49335.000000\n",
       "mean       4.338218   1999.642059     57.551191      1.022995     -0.139069\n",
       "std        2.307135      7.935081     18.349595      0.587244      0.619135\n",
       "min        1.000000   1994.000000     25.000000      0.012390     -4.390870\n",
       "25%        2.000000   1995.000000     40.000000      0.632637     -0.457858\n",
       "50%        4.000000   1997.000000     62.000000      0.896917     -0.108792\n",
       "75%        6.000000   1999.000000     73.000000      1.256659      0.228456\n",
       "max        8.000000   2021.000000     85.000000      4.053713      1.399633"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[(df['age']>=25) & (df['age']<=85)]\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These are the functions for estimating parameters by minimum distance. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vech(cov):\n",
    "\tT = cov.shape[0]\n",
    "\tv = []\n",
    "\tfor i in range(T):\n",
    "\t\tfor j in range(i,T):\n",
    "\t\t\tv.append(cov[j,i])\n",
    "\treturn np.array(v)\n",
    "def mdfunc(theta,cov,weight):\n",
    "\trho = theta[0]\n",
    "\tsige = theta[1]\n",
    "\tsigv = theta[2]\n",
    "\tT = cov.shape[0]\n",
    "\tcov_true = np.zeros((T,T))\n",
    "\tfor i in range(T):\n",
    "\t\tfor j in range(T):\n",
    "\t\t\tcov_true[i,j] = (i==j)*sigv + (rho**(np.abs(i-j)))*sige/(1.0-rho**2)\n",
    "\td = vech(cov) - vech(cov_true)\n",
    "\tfunc = d.reshape((1,len(d))) @ weight @ d.reshape((len(d),1))\n",
    "\treturn func[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(seed=1234)\n",
    "def get_cov(data,reps):\n",
    "\tT = len(data.columns)\n",
    "\tcov = data.cov().to_numpy()\n",
    "\tvcov = vech(cov)\n",
    "\tweight = vcov.reshape((vcov.shape[0],1))@vcov.reshape((1,vcov.shape[0]))\n",
    "\tweight = np.zeros((weight.shape[0],weight.shape[1]))\n",
    "\tfor r in range(reps):\n",
    "\t\trcov = data.sample(n=len(data),replace=True).cov().to_numpy()\n",
    "\t\tvrcov = vech(rcov)\n",
    "\t\tweight += vrcov.reshape((vrcov.shape[0],1))@vrcov.reshape((1,vrcov.shape[0]))\n",
    "\tweight = np.linalg.inv(weight/reps)\n",
    "\treturn cov, weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pars(co):\n",
    "\tdata = df.loc[(df['co']==co) & (df['logY'].isna()==False),:]\n",
    "\tdata = data.reset_index()\n",
    "\tdata = data.loc[:,['pid','wave','age','logY']]\n",
    "\tmodel = ols('logY ~ C(age)',data=data).fit()\n",
    "\tdata.loc[:,'e'] = model.resid.to_list()\n",
    "\tdata = data.loc[:,['pid','wave','e']]\n",
    "\tdata = pd.pivot(data,index='pid',columns='wave')\n",
    "\tcov  = data.cov().to_numpy()\n",
    "\tweight = np.eye(vech(cov).shape[0])\n",
    "\tf = partial(mdfunc,cov=cov,weight=weight)\n",
    "\titheta = np.array([0.9,0.02,0.02])\n",
    "\topt = minimize(f,x0=itheta)\n",
    "\treturn opt.x\n",
    "\t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now estimate parameters for each countries. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DE</th>\n",
       "      <th>DK</th>\n",
       "      <th>FR</th>\n",
       "      <th>NL</th>\n",
       "      <th>IT</th>\n",
       "      <th>SP</th>\n",
       "      <th>SE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rho</th>\n",
       "      <td>0.920811</td>\n",
       "      <td>0.912798</td>\n",
       "      <td>0.967463</td>\n",
       "      <td>0.943292</td>\n",
       "      <td>0.937389</td>\n",
       "      <td>0.972208</td>\n",
       "      <td>0.912798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sige</th>\n",
       "      <td>0.038995</td>\n",
       "      <td>0.015786</td>\n",
       "      <td>0.013734</td>\n",
       "      <td>0.018716</td>\n",
       "      <td>0.028802</td>\n",
       "      <td>0.014819</td>\n",
       "      <td>0.015786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sigv</th>\n",
       "      <td>0.076511</td>\n",
       "      <td>0.070812</td>\n",
       "      <td>0.133274</td>\n",
       "      <td>0.118675</td>\n",
       "      <td>0.157314</td>\n",
       "      <td>0.263140</td>\n",
       "      <td>0.070812</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            DE        DK        FR        NL        IT        SP        SE\n",
       "rho   0.920811  0.912798  0.967463  0.943292  0.937389  0.972208  0.912798\n",
       "sige  0.038995  0.015786  0.013734  0.018716  0.028802  0.014819  0.015786\n",
       "sigv  0.076511  0.070812  0.133274  0.118675  0.157314  0.263140  0.070812"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countries = ['DE','DK','FR','NL','IT','SP']\n",
    "table = pd.DataFrame(index=['rho','sige','sigv'],columns=countries)\n",
    "for co in countries:\n",
    "\ttable[co] = get_pars(co)\n",
    "table['SE'] = table.loc[:,'DK']\n",
    "table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We re-arange order for presentation in paper. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DE</th>\n",
       "      <th>DK</th>\n",
       "      <th>FR</th>\n",
       "      <th>IT</th>\n",
       "      <th>NL</th>\n",
       "      <th>SE</th>\n",
       "      <th>SP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rho</th>\n",
       "      <td>0.920811</td>\n",
       "      <td>0.912798</td>\n",
       "      <td>0.967463</td>\n",
       "      <td>0.937389</td>\n",
       "      <td>0.943292</td>\n",
       "      <td>0.912798</td>\n",
       "      <td>0.972208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sige</th>\n",
       "      <td>0.038995</td>\n",
       "      <td>0.015786</td>\n",
       "      <td>0.013734</td>\n",
       "      <td>0.028802</td>\n",
       "      <td>0.018716</td>\n",
       "      <td>0.015786</td>\n",
       "      <td>0.014819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sigv</th>\n",
       "      <td>0.076511</td>\n",
       "      <td>0.070812</td>\n",
       "      <td>0.133274</td>\n",
       "      <td>0.157314</td>\n",
       "      <td>0.118675</td>\n",
       "      <td>0.070812</td>\n",
       "      <td>0.263140</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            DE        DK        FR        IT        NL        SE        SP\n",
       "rho   0.920811  0.912798  0.967463  0.937389  0.943292  0.912798  0.972208\n",
       "sige  0.038995  0.015786  0.013734  0.028802  0.018716  0.015786  0.014819\n",
       "sigv  0.076511  0.070812  0.133274  0.157314  0.118675  0.070812  0.263140"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table = table.loc[:,['DE','DK','FR','IT','NL','SE','SP']]\n",
    "table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now repeat the same for the PSID (US) data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>co</th>\n",
       "      <th>pid</th>\n",
       "      <th>year</th>\n",
       "      <th>age</th>\n",
       "      <th>Y</th>\n",
       "      <th>logY</th>\n",
       "      <th>wave</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>US</td>\n",
       "      <td>4001</td>\n",
       "      <td>1989</td>\n",
       "      <td>67.0</td>\n",
       "      <td>0.720443</td>\n",
       "      <td>-0.327888</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>US</td>\n",
       "      <td>4003</td>\n",
       "      <td>1989</td>\n",
       "      <td>38.0</td>\n",
       "      <td>0.900614</td>\n",
       "      <td>-0.104679</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>US</td>\n",
       "      <td>4008</td>\n",
       "      <td>1989</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.684175</td>\n",
       "      <td>-0.379542</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>US</td>\n",
       "      <td>4170</td>\n",
       "      <td>1989</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.789620</td>\n",
       "      <td>-0.236204</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>US</td>\n",
       "      <td>4172</td>\n",
       "      <td>1989</td>\n",
       "      <td>40.0</td>\n",
       "      <td>1.564385</td>\n",
       "      <td>0.447493</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    co   pid  year   age         Y      logY  wave\n",
       "0   US  4001  1989  67.0  0.720443 -0.327888     1\n",
       "2   US  4003  1989  38.0  0.900614 -0.104679     1\n",
       "6   US  4008  1989  24.0  0.684175 -0.379542     1\n",
       "15  US  4170  1989  34.0  0.789620 -0.236204     1\n",
       "16  US  4172  1989  40.0  1.564385  0.447493     1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_pickle('../data_sources/psid/psid_incomes.pkl')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>year</th>\n",
       "      <th>age</th>\n",
       "      <th>Y</th>\n",
       "      <th>logY</th>\n",
       "      <th>wave</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>6.455200e+04</td>\n",
       "      <td>64552.000000</td>\n",
       "      <td>64552.000000</td>\n",
       "      <td>64552.000000</td>\n",
       "      <td>64552.000000</td>\n",
       "      <td>64552.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.497481e+06</td>\n",
       "      <td>1993.055103</td>\n",
       "      <td>45.855930</td>\n",
       "      <td>0.992746</td>\n",
       "      <td>-0.280264</td>\n",
       "      <td>5.055103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.304755e+06</td>\n",
       "      <td>2.518986</td>\n",
       "      <td>15.271426</td>\n",
       "      <td>0.685772</td>\n",
       "      <td>0.802517</td>\n",
       "      <td>2.518986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>4.001000e+03</td>\n",
       "      <td>1989.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>0.093259</td>\n",
       "      <td>-2.372372</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.398002e+06</td>\n",
       "      <td>1991.000000</td>\n",
       "      <td>34.000000</td>\n",
       "      <td>0.467730</td>\n",
       "      <td>-0.759865</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.665005e+06</td>\n",
       "      <td>1993.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>0.845519</td>\n",
       "      <td>-0.167804</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.815001e+06</td>\n",
       "      <td>1995.000000</td>\n",
       "      <td>56.000000</td>\n",
       "      <td>1.358952</td>\n",
       "      <td>0.306714</td>\n",
       "      <td>7.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>6.872177e+06</td>\n",
       "      <td>1997.000000</td>\n",
       "      <td>85.000000</td>\n",
       "      <td>2.982882</td>\n",
       "      <td>1.092890</td>\n",
       "      <td>9.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                pid          year           age             Y          logY  \\\n",
       "count  6.455200e+04  64552.000000  64552.000000  64552.000000  64552.000000   \n",
       "mean   3.497481e+06   1993.055103     45.855930      0.992746     -0.280264   \n",
       "std    2.304755e+06      2.518986     15.271426      0.685772      0.802517   \n",
       "min    4.001000e+03   1989.000000     25.000000      0.093259     -2.372372   \n",
       "25%    1.398002e+06   1991.000000     34.000000      0.467730     -0.759865   \n",
       "50%    2.665005e+06   1993.000000     42.000000      0.845519     -0.167804   \n",
       "75%    5.815001e+06   1995.000000     56.000000      1.358952      0.306714   \n",
       "max    6.872177e+06   1997.000000     85.000000      2.982882      1.092890   \n",
       "\n",
       "               wave  \n",
       "count  64552.000000  \n",
       "mean       5.055103  \n",
       "std        2.518986  \n",
       "min        1.000000  \n",
       "25%        3.000000  \n",
       "50%        5.000000  \n",
       "75%        7.000000  \n",
       "max        9.000000  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.loc[(df['age']>=25) & (df['age']<=85),:]\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "pars = get_pars('US')\n",
    "table.loc[:,'US'] = pars"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We add to the table the stationary variance of the process. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DE</th>\n",
       "      <th>DK</th>\n",
       "      <th>FR</th>\n",
       "      <th>IT</th>\n",
       "      <th>NL</th>\n",
       "      <th>SE</th>\n",
       "      <th>SP</th>\n",
       "      <th>US</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rho</th>\n",
       "      <td>0.920811</td>\n",
       "      <td>0.912798</td>\n",
       "      <td>0.967463</td>\n",
       "      <td>0.937389</td>\n",
       "      <td>0.943292</td>\n",
       "      <td>0.912798</td>\n",
       "      <td>0.972208</td>\n",
       "      <td>0.958992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sige</th>\n",
       "      <td>0.038995</td>\n",
       "      <td>0.015786</td>\n",
       "      <td>0.013734</td>\n",
       "      <td>0.028802</td>\n",
       "      <td>0.018716</td>\n",
       "      <td>0.015786</td>\n",
       "      <td>0.014819</td>\n",
       "      <td>0.039076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sigv</th>\n",
       "      <td>0.076511</td>\n",
       "      <td>0.070812</td>\n",
       "      <td>0.133274</td>\n",
       "      <td>0.157314</td>\n",
       "      <td>0.118675</td>\n",
       "      <td>0.070812</td>\n",
       "      <td>0.263140</td>\n",
       "      <td>0.097208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sigs</th>\n",
       "      <td>0.256367</td>\n",
       "      <td>0.094643</td>\n",
       "      <td>0.214538</td>\n",
       "      <td>0.237439</td>\n",
       "      <td>0.169834</td>\n",
       "      <td>0.094643</td>\n",
       "      <td>0.270357</td>\n",
       "      <td>0.486429</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            DE        DK        FR        IT        NL        SE        SP  \\\n",
       "rho   0.920811  0.912798  0.967463  0.937389  0.943292  0.912798  0.972208   \n",
       "sige  0.038995  0.015786  0.013734  0.028802  0.018716  0.015786  0.014819   \n",
       "sigv  0.076511  0.070812  0.133274  0.157314  0.118675  0.070812  0.263140   \n",
       "sigs  0.256367  0.094643  0.214538  0.237439  0.169834  0.094643  0.270357   \n",
       "\n",
       "            US  \n",
       "rho   0.958992  \n",
       "sige  0.039076  \n",
       "sigv  0.097208  \n",
       "sigs  0.486429  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table.loc['sigs',:] = table.loc['sige',:]/(1.0-table.loc['rho',:]**2)\n",
    "table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is Table 5 in the paper. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "table.round(3).to_latex('../tables/table_5_income_shocks.tex')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving parameters for input in the model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = table.loc[['rho','sige'],:]\n",
    "params.to_pickle('../model/params/income_shocks.pkl')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "cf2a50979671a58939829e6829efb726aa5da11149213b77bd50351f899d04fb"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
